# coding=utf-8
# 学习单位  : 郑州大学
# @Author  : 铭同学
# @Time    : 2021/9/29 22:10
# @Software: PyCharm


import re
from urllib import request,error    #指定URL，获取网页数据
import requests
import pandas as pd
from bs4 import BeautifulSoup # 可以利用selecto解析
from lxml import etree  # 可以利用xpath解析

# page=  可以直接锁定某一页  1-29625但是只能访问到200页
baseurl = 'https://www.vesselfinder.com/vessels?page='



#获取指定一个URL的网页内容
def askURL(url):
    #模拟浏览器头部信息，向豆瓣服务器发送消息
    headers = {
        #用户代理，告诉豆瓣服务器，访问者时什么类型的机器、浏览器
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.131 Safari/537.36 Edg/92.0.902.73"
    }
    req = request.Request(url,headers=headers)
    #定义空字符串html
    html = ""
    try:
        response = request.urlopen(req)
        # 这里只保留文字？
        # html = response.read().decode("utf-8")
        html = response.read()
    except error.URLError as e:
        if hasattr(e,"code"):
            print(e.code)
        if hasattr(e,"reason"):
            print(e.reason)

    return html

# 用来获取多个网页
def lxmlgetData():

    pic_list = []
    name_list = []
    # 可以获取200页-200
    for i in range(200):
        print(f'获取第{i+1}页的信息...')
        url = baseurl + str(i+1)
        html = askURL(url)  #保存获取到的网页源码
        html=etree.HTML(html,etree.HTMLParser())    # 转换为etree对象，可以使用xpath进行定位

        # 获取图片信息（不是本地信息，抓不到哦）最后要加上@data-src，取到其云端链接属性值
        # 云端图片链接无规律，还是要进行匹配和解析  data-src字段
        pic_result = html.xpath("/html/body/div/div/main/div/section/table/tbody/tr/td[1]/a/img/@data-src")
        for pic_url in pic_result:
            # 获取每个链接后，再对每个链接进行抓取，获得图片实体(只能获取链接)
            # 如果采集下来的URL不是以图片格式（jpg,png,gif）结尾（3.jfif），则有可能不能转换，可能是网站对此图片链接进行加密仅支持在线查看。
            askURL(pic_url)
            pic_list.append(pic_url)

        # result形成了一个列表 /html/body/div/div/main/div/section/table/tbody/tr[2]/td[2]/a/div[2]/div[2]
        name_result = html.xpath("/html/body/div/div/main/div/section/table/tbody/tr/td[2]/a/div[2]/div[2]")
        for i in name_result:
            name_list.append(i.text)

    print('信息获取完成！！！')
    return [name_list,pic_list]


# 可将图片的link直接存储为图片格式到本地
def get_pictures(url,path):
    headers={
        'user-agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.25 Safari/537.36 Core/1.70.3861.400 QQBrowser/10.7.4313.400'}
    re=requests.get(url,headers=headers)
    print(re.status_code)   #查看请求状态，返回200说明正常
    with open(path, 'wb') as f: #把图片数据写入本地，wb表示二进制储存
        for chunk in re.iter_content(chunk_size=128):
            f.write(chunk)


# 获取图片并保存图片到本地（前提是已经获取图片的link）
def get_pictures_(pic_urls):
    urls=pic_urls #获取当前页面所有图片的url
    for i in range(len(urls)):#批量爬取图片
        print(f'正在爬取第{i+1}条...')
        url=urls[i]
        path='./pics/Ship'+str(i)+'.jpg'
        get_pictures(url,path)


def list_to_df():
    # 列表
    data = lxmlgetData()
    # data = [['Offshore Support Vessel', 'Crane Ship', 'FSO', 'FSO', 'Container Ship',
    #          'Container Ship', 'Container Ship', 'Container Ship', 'Container Ship', 'Container Ship', 'Container Ship', 'Container Ship', 'Container Ship', 'Container Ship', 'Crude Oil Tanker', 'FSO', 'Container Ship', 'Container Ship', 'Container Ship', 'Container Ship', 'Container Ship', 'Container Ship', 'Container Ship', 'Container Ship', 'Container Ship', 'Container Ship', 'Container Ship', 'Container Ship', 'Container Ship', 'Container Ship', 'Container Ship', 'Container Ship', 'Container Ship', 'Container Ship', 'Container Ship', 'Container Ship', 'Container Ship', 'Container Ship', 'Container Ship', 'Container Ship', 'Container Ship', 'Container Ship', 'Passenger (Cruise) Ship', 'Passenger (Cruise) Ship', 'Passenger (Cruise) Ship', 'Passenger (Cruise) Ship', 'Offshore Support Vessel', 'Container Ship', 'Container Ship', 'Container Ship', 'Container Ship', 'Container Ship', 'Container Ship', 'Container Ship', 'Container Ship', 'Container Ship', 'Container Ship', 'Container Ship', 'Container Ship', 'Container Ship'], ['https://static.vesselfinder.net/ship-photo/9648714-0-c926e6a381efbfb0bcee0694ffe22c52/3', 'https://static.vesselfinder.net/ship-photo/9593505-249110000-d6ef3c352e2c4d216196ffb5c0f7bbe2/3', 'https://static.vesselfinder.net/ship-photo/9224764-538002388-1bdab74e4cc95d12f71641b9b5717ad9/3', 'https://static.vesselfinder.net/ship-photo/9224752-538003426-ddab7cede18abfb5495fb7053a63ed73/3', 'https://static.vesselfinder.net/ship-photo/9839155-228397700-a4821244c918dd335a478bf15a8e8944/3', 'https://static.vesselfinder.net/ship-photo/9839179-228386700-9905a792c57d0b900d2d28ef6ce58d8d/3', 'https://static.vesselfinder.net/ship-photo/9839181-228394600-e347335bf70da389430d7f4584354bf6/3', 'https://static.vesselfinder.net/ship-photo/9839131-228386800-7c82cf66df383abea1735e92bf9d5fd0/3', 'https://static.vesselfinder.net/ship-photo/9839208-228401800-93262d50da7a7e31dd9938c6c148393f/3', 'https://static.vesselfinder.net/ship-photo/9839167-228402900-464191c0a8a45e62b37867781150d82b/3', 'https://static.vesselfinder.net/ship-photo/9839210-228038360-c80bbd0e66fb9e5219716770f377f787/3', 'https://static.vesselfinder.net/ship-photo/9839143-228394900-20fcdf61682c6686aedd00eb5a596cc1/3', 'https://static.vesselfinder.net/ship-photo/9839193-228397600-1cf2cb951b16af24d531c592a85e2ff5/3', 'https://static.vesselfinder.net/ship-photo/9893890-352986146-ce4356c793794d37ad82cb15a1109cbd/3', 'https://static.vesselfinder.net/ship-photo/9235268-205408000-2647cf04ddde5c56f52146f1831e2a27/3', 'https://static.vesselfinder.net/ship-photo/9246633-538002371-e0fa51c127732277811de4678640bae9/3', 'https://static.vesselfinder.net/ship-photo/9839454-356432000-dae44a22608c8f6165f52f0880d4b63d/3', 'https://static.vesselfinder.net/ship-photo/9839466-355798000-1f248457c5dfb12c1c42a3f244d51bee/3', 'https://static.vesselfinder.net/ship-photo/9839430-372003000-8504bad7bd4373ea10e99743250b2235/3', 'https://static.vesselfinder.net/ship-photo/9839480-352269000-263b2be303d83f5429226db79c6aa266/3', 'https://static.vesselfinder.net/ship-photo/9839442-371047000-51ffcb2e3d1a3682f126e4659b6b92a6/3', 'https://static.vesselfinder.net/ship-photo/9839478-372729000-0fd8cd0d0ed928e2df7346114ab04a3c/3', 'https://static.vesselfinder.net/ship-photo/9868364-351404000-864b4b16b17c3361438262a66f9b598b/3', 'https://static.vesselfinder.net/ship-photo/9868352-351127000-bf8392b041967ece1e4399497d70c845/3', 'https://static.vesselfinder.net/ship-photo/9868340-357770000-cbb3f991a493807369b157d4a668d6b3/3', 'https://static.vesselfinder.net/ship-photo/9868338-351246000-3fe35f28e43f69bf27f80c091c3963dc/3', 'https://static.vesselfinder.net/ship-photo/9868326-371319000-803f81c12b25edbdff26d30527e8c8b5/3', 'https://static.vesselfinder.net/ship-photo/9897004-636020598-d4334e0cf0e4822487fa60e695c81f3c/3', 'https://static.vesselfinder.net/ship-photo/9896983-636020542-dd3796912f5fe2b58d9c0e26224627a3/3', 'https://static.vesselfinder.net/ship-photo/9896995-636020543-d261cf3e84717597d79d704245745343/3', 'https://static.vesselfinder.net/ship-photo/9839301-370711000-74a539b9a93dbb03f0cee223b1559c3f/3', 'https://static.vesselfinder.net/ship-photo/9839260-351862000-64ee48e2148e6fb5c76c09fde0727d92/3', 'https://static.vesselfinder.net/ship-photo/9839284-374607000-97b0c203aff56706d72aab072c78a00d/3', 'https://static.vesselfinder.net/ship-photo/9839272-353590000-499b8cdc3a9e2d96a5dabb73f390bc37/3', 'https://static.vesselfinder.net/ship-photo/9839296-356234000-8d43e67bf4362f312e4594553b898422/3', 'https://static.vesselfinder.net/ship-photo/9863302-356712000-4f93a7e579eb9289a61553d37ee5b1c5/3', 'https://static.vesselfinder.net/ship-photo/9863326-374241000-c78cde4dce7b44ab3d8116fff5223e0c/3', 'https://static.vesselfinder.net/ship-photo/9868314-352260000-833053b1dfd8b28ef996edfc22843589/3', 'https://static.vesselfinder.net/ship-photo/9863314-356429000-c15e509e5d76f6b3ad1a23db7ca29c21/3', 'https://static.vesselfinder.net/ship-photo/9863340-357992000-d8fb4b4c68547ffdcff604bf0ce00b38/3', 'https://static.vesselfinder.net/ship-photo/9863297-351297000-e067e3bfd5b946920340d738c14fe57b/3', 'https://static.vesselfinder.net/ship-photo/9863338-374102000-ff0cb58435382883153f6cac9e5a1c79/3', 'https://static.vesselfinder.net/ship-photo/9744001-311000660-4d195caec7abee75a96330b75ea1fb9a/3', 'https://static.vesselfinder.net/ship-photo/9682875-311000396-2e01a1aaeeed6804299294f2ec1e81fa/3', 'https://static.vesselfinder.net/ship-photo/9383936-311020600-89bee89898761617892f92f22f053168/3', 'https://static.vesselfinder.net/ship-photo/9383948-311020700-c05a2dc32f566f041eb6c423ded43c26/3', 'https://static.vesselfinder.net/ship-photo/9695896-372750000-4c68574676bd670a1f5891cc5a313aee/3', 'https://static.vesselfinder.net/ship-photo/9786839-636019234-2ba9f0d64dcf763adc84d1debda39246/3', 'https://static.vesselfinder.net/ship-photo/9786841-354977000-d6581c9b9682549ba3ee9840d70ff39c/3', 'https://static.vesselfinder.net/ship-photo/9832729-371829000-83941c65a4b3ce1fff18fbf4031eb602/3', 'https://static.vesselfinder.net/ship-photo/9832717-356582000-11c47eac8dd0fa7b5e20d88345646a75/3', 'https://static.vesselfinder.net/ship-photo/9786815-370121000-ede532803862710ff8958b7c25d3b867/3', 'https://static.vesselfinder.net/ship-photo/9786827-563068900-0e3b7ca5b4d518a832f514a4c9d481da/3', 'https://static.vesselfinder.net/ship-photo/9776432-248794000-a9f3247c40ced4cbdb29e743e3304e93/3', 'https://static.vesselfinder.net/ship-photo/9776420-248758000-f564aa100b8afaf451e2a963c9752c09/3', 'https://static.vesselfinder.net/ship-photo/9776418-228098700-a4c76a91ea60972da5df037ea18203be/3', 'https://static.vesselfinder.net/ship-photo/9820855-357463000-fe8218b02b081a9d16c2acfae1bfc356/3', 'https://static.vesselfinder.net/ship-photo/9811012-354654000-9eb81b2d4c54a7deaa4054fa33eca6c2/3', 'https://static.vesselfinder.net/ship-photo/9810991-371308000-28f30f36a0f3c44874266037a7ca6b47/3', 'https://static.vesselfinder.net/ship-photo/9811000-353136000-ce939b7d94cba3939b8822079df7ccf1/3']]
    data = list(map(list, zip(*data)))	#列表转置，行列转换

    names = ['Ship_name', 'Pic_link']
    df = pd.DataFrame(data,columns=names)
    # df = pd.DataFrame(data).T


    # 字典
    # dict = {
    #     'MCC3':data[0],
    #     '国家外文名':data[1],
    #     '国家码':data[2],
    #     '国家中文名':data[3]
    # }
    #
    # #将字典转换成为数据框
    # df = pd.DataFrame(dict)


    # 保存到本地csv
    df.to_csv("Shipinfo.csv", encoding='utf-8', index=False)
    print('信息保存完成！！！')



if __name__ == '__main__':
    pass
    # getData()
    # list_to_df()
    # data = lxmlgetData()  # 得到两个列表组成的列表
    # print(data)


    # 获取一次html保存起来进行解析
    # html = askURL(url)
    # print(html)


    # 保存图片到本地
    # data = pd.read_csv("Shipinfo.csv", encoding='utf-8')["Pic_link"]
    # get_pictures_(data)







